<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - optimization_least_squares.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2010  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_OPTIMIZATION_LEAST_SQuARES_H___
<font color='#0000FF'>#define</font> DLIB_OPTIMIZATION_LEAST_SQuARES_H___

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../matrix.h.html'>../matrix.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='optimization_trust_region.h.html'>optimization_trust_region.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='optimization_least_squares_abstract.h.html'>optimization_least_squares_abstract.h</a>"

<font color='#0000FF'>namespace</font> dlib
<b>{</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> column_vector_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='least_squares_function_model'></a>least_squares_function_model</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='least_squares_function_model'></a>least_squares_function_model</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f_,
            <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der_,
            <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list_
        <font face='Lucida Console'>)</font> : f<font face='Lucida Console'>(</font>f_<font face='Lucida Console'>)</font>, der<font face='Lucida Console'>(</font>der_<font face='Lucida Console'>)</font>, list<font face='Lucida Console'>(</font>list_<font face='Lucida Console'>)</font> 
        <b>{</b>
            S <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            last_f <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            last_f2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            r.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f;
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der;
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list;

        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::type type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::mem_manager_type mem_manager_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::layout_type layout_type;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>long</u></font> NR <font color='#5555FF'>=</font> column_vector_type::NR;

        <font color='#0000FF'>typedef</font> column_vector_type column_vector;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>type,NR,NR,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> general_matrix;


        type <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font> 
            <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            type result <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> type temp <font color='#5555FF'>=</font> <font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#BB00BB'>list</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>, x<font face='Lucida Console'>)</font>;
                <font color='#009900'>// save the residual for later
</font>                <font color='#BB00BB'>r</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> temp;
                result <font color='#5555FF'>+</font><font color='#5555FF'>=</font> temp<font color='#5555FF'>*</font>temp;
            <b>}</b>

            last_f <font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font color='#5555FF'>*</font>result;
            <font color='#0000FF'>return</font> <font color='#979000'>0.5</font><font color='#5555FF'>*</font>result;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='get_derivative_and_hessian'></a>get_derivative_and_hessian</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x,
            column_vector<font color='#5555FF'>&amp;</font> d,
            general_matrix<font color='#5555FF'>&amp;</font> h
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            J.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// compute the Jacobian
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>set_rowm</font><font face='Lucida Console'>(</font>J,i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>der</font><font face='Lucida Console'>(</font><font color='#BB00BB'>list</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>, x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#009900'>// Compute the Levenberg-Marquardt gradient and hessian
</font>            d <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>J<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>r;
            h <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>J<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>J;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>S.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                S.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                S <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>

            <font color='#009900'>// If this isn't the first iteration then check if using
</font>            <font color='#009900'>// a quasi-newton update helps things.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>last_r.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>

                s <font color='#5555FF'>=</font> x <font color='#5555FF'>-</font> last_x;
                y <font color='#5555FF'>=</font> d <font color='#5555FF'>-</font> last_d;
                yy <font color='#5555FF'>=</font> d <font color='#5555FF'>-</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>last_J<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>r;

                <font color='#0000FF'>const</font> type ys <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>y<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s;
                vtemp <font color='#5555FF'>=</font> yy <font color='#5555FF'>-</font> S<font color='#5555FF'>*</font>s;
                <font color='#0000FF'>const</font> type temp2 <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>S<font color='#5555FF'>*</font>s<font face='Lucida Console'>)</font>;
                type scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>temp2 <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> ? std::min<font color='#5555FF'>&lt;</font>type<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>s,yy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>temp2<font face='Lucida Console'>)</font>  :  <font color='#979000'>1</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>ys <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    S <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>S <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>vtemp<font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>y<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> y<font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>vtemp<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>ys<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>vtemp,s<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>ys<font color='#5555FF'>/</font>ys<font color='#5555FF'>*</font>y<font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>y<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    S <font color='#5555FF'>*</font><font color='#5555FF'>=</font> scale;

                <font color='#009900'>// check how well both the models fit the last change we saw in f()
</font>                <font color='#0000FF'>const</font> type measured_delta <font color='#5555FF'>=</font> last_f2 <font color='#5555FF'>-</font> last_f;
                s <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>s;
                <font color='#0000FF'>const</font> type h_predicted_delta <font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>h<font color='#5555FF'>*</font>s <font color='#5555FF'>+</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>d<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s;
                <font color='#0000FF'>const</font> type s_predicted_delta <font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>h<font color='#5555FF'>+</font>S<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s <font color='#5555FF'>+</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>d<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s;

                <font color='#0000FF'>const</font> type h_error <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>h_predicted_delta<font color='#5555FF'>/</font>measured_delta<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> type s_error <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>s_predicted_delta<font color='#5555FF'>/</font>measured_delta<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s_error <font color='#5555FF'>&lt;</font> h_error <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> h_error <font color='#5555FF'>&gt;</font> <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    h <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>make_symmetric</font><font face='Lucida Console'>(</font>S<font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s_error <font color='#5555FF'>&gt;</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    S <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>

                <font color='#009900'>// put r into last_r
</font>                r.<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>last_r<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#009900'>// put r into last_r
</font>                last_r <font color='#5555FF'>=</font> r;
            <b>}</b>

            J.<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>last_J<font face='Lucida Console'>)</font>;
            last_x <font color='#5555FF'>=</font> x;
            last_d <font color='#5555FF'>=</font> d;

            last_f2 <font color='#5555FF'>=</font> last_f;
        <b>}</b>

        <font color='#0000FF'>mutable</font> type last_f;   <font color='#009900'>// value of function we saw in last operator()
</font>        <font color='#0000FF'>mutable</font> type last_f2;  <font color='#009900'>// value of last_f we saw in get_derivative_and_hessian()
</font>        <font color='#0000FF'>mutable</font> matrix<font color='#5555FF'>&lt;</font>type,<font color='#979000'>0</font>,<font color='#979000'>1</font>,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> r;
        <font color='#0000FF'>mutable</font> column_vector vtemp;
        <font color='#0000FF'>mutable</font> column_vector s, y, yy;

        <font color='#0000FF'>mutable</font> general_matrix S;
        <font color='#0000FF'>mutable</font> column_vector last_x;
        <font color='#0000FF'>mutable</font> column_vector last_d;
        <font color='#0000FF'>mutable</font> matrix<font color='#5555FF'>&lt;</font>type,<font color='#979000'>0</font>,<font color='#979000'>1</font>,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> last_r;
        <font color='#0000FF'>mutable</font> matrix<font color='#5555FF'>&lt;</font>type,<font color='#979000'>0</font>,NR,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> last_J;
        <font color='#0000FF'>mutable</font> matrix<font color='#5555FF'>&lt;</font>type,<font color='#979000'>0</font>,NR,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> J;
    <b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> column_vector_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type
        <font color='#5555FF'>&gt;</font>
    least_squares_function_model<font color='#5555FF'>&lt;</font>column_vector_type,funct_type,funct_der_type,vector_type<font color='#5555FF'>&gt;</font> <b><a name='least_squares_model'></a>least_squares_model</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f,
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der,
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> least_squares_function_model<font color='#5555FF'>&lt;</font>column_vector_type,funct_type,funct_der_type,vector_type<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>f,der,list<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> stop_strategy_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type,
        <font color='#0000FF'>typename</font> T
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='solve_least_squares'></a>solve_least_squares</b> <font face='Lucida Console'>(</font>
        stop_strategy_type stop_strategy,
        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f,
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der,
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list,
        T<font color='#5555FF'>&amp;</font> x, 
        <font color='#0000FF'><u>double</u></font> radius <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// The starting point (i.e. x) must be a column vector.  
</font>        <font color='#BB00BB'>COMPILE_TIME_ASSERT</font><font face='Lucida Console'>(</font>T::NC <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure requires clause is not broken
</font>        <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_vector</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                    <font color='#BB00BB'>is_col_vector</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> radius <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>,
            "<font color='#CC0000'>\t double solve_least_squares()</font>"
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t invalid arguments were given to this function</font>"
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t is_vector(list):  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>is_vector</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t list.size():      </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t is_col_vector(x): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>is_col_vector</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t radius:           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> radius
            <font face='Lucida Console'>)</font>;

        <font color='#0000FF'>return</font> <font color='#BB00BB'>find_min_trust_region</font><font face='Lucida Console'>(</font>stop_strategy,
                                     least_squares_model<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>f, der, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
                                     x, 
                                     radius<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> column_vector_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='least_squares_lm_function_model'></a>least_squares_lm_function_model</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='least_squares_lm_function_model'></a>least_squares_lm_function_model</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f_,
            <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der_,
            <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list_
        <font face='Lucida Console'>)</font> : f<font face='Lucida Console'>(</font>f_<font face='Lucida Console'>)</font>, der<font face='Lucida Console'>(</font>der_<font face='Lucida Console'>)</font>, list<font face='Lucida Console'>(</font>list_<font face='Lucida Console'>)</font> 
        <b>{</b>
            r.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f;
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der;
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list;

        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::type type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::mem_manager_type mem_manager_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> column_vector_type::layout_type layout_type;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>long</u></font> NR <font color='#5555FF'>=</font> column_vector_type::NR;

        <font color='#0000FF'>typedef</font> column_vector_type column_vector;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>type,NR,NR,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> general_matrix;

        <font color='#0000FF'>mutable</font> matrix<font color='#5555FF'>&lt;</font>type,<font color='#979000'>0</font>,<font color='#979000'>1</font>,mem_manager_type,layout_type<font color='#5555FF'>&gt;</font> r;
        <font color='#0000FF'>mutable</font> column_vector vtemp;

        type <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font> 
            <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            type result <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> type temp <font color='#5555FF'>=</font> <font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#BB00BB'>list</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>, x<font face='Lucida Console'>)</font>;
                <font color='#009900'>// save the residual for later
</font>                <font color='#BB00BB'>r</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> temp;
                result <font color='#5555FF'>+</font><font color='#5555FF'>=</font> temp<font color='#5555FF'>*</font>temp;
            <b>}</b>

            <font color='#0000FF'>return</font> <font color='#979000'>0.5</font><font color='#5555FF'>*</font>result;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='get_derivative_and_hessian'></a>get_derivative_and_hessian</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x,
            column_vector<font color='#5555FF'>&amp;</font> d,
            general_matrix<font color='#5555FF'>&amp;</font> h
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            d <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            h <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                vtemp <font color='#5555FF'>=</font> <font color='#BB00BB'>der</font><font face='Lucida Console'>(</font><font color='#BB00BB'>list</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>, x<font face='Lucida Console'>)</font>; 
                d <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>r</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>vtemp;
                h <font color='#5555FF'>+</font><font color='#5555FF'>=</font> vtemp<font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>vtemp<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

    <b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> column_vector_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type
        <font color='#5555FF'>&gt;</font>
    least_squares_lm_function_model<font color='#5555FF'>&lt;</font>column_vector_type,funct_type,funct_der_type,vector_type<font color='#5555FF'>&gt;</font> <b><a name='least_squares_lm_model'></a>least_squares_lm_model</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f,
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der,
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> least_squares_lm_function_model<font color='#5555FF'>&lt;</font>column_vector_type,funct_type,funct_der_type,vector_type<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>f,der,list<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> stop_strategy_type,
        <font color='#0000FF'>typename</font> funct_type,
        <font color='#0000FF'>typename</font> funct_der_type,
        <font color='#0000FF'>typename</font> vector_type,
        <font color='#0000FF'>typename</font> T
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='solve_least_squares_lm'></a>solve_least_squares_lm</b> <font face='Lucida Console'>(</font>
        stop_strategy_type stop_strategy,
        <font color='#0000FF'>const</font> funct_type<font color='#5555FF'>&amp;</font> f,
        <font color='#0000FF'>const</font> funct_der_type<font color='#5555FF'>&amp;</font> der,
        <font color='#0000FF'>const</font> vector_type<font color='#5555FF'>&amp;</font> list,
        T<font color='#5555FF'>&amp;</font> x, 
        <font color='#0000FF'><u>double</u></font> radius <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// The starting point (i.e. x) must be a column vector.  
</font>        <font color='#BB00BB'>COMPILE_TIME_ASSERT</font><font face='Lucida Console'>(</font>T::NC <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure requires clause is not broken
</font>        <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_vector</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                    <font color='#BB00BB'>is_col_vector</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> radius <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>,
            "<font color='#CC0000'>\t double solve_least_squares_lm()</font>"
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t invalid arguments were given to this function</font>"
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t is_vector(list):  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>is_vector</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t list.size():      </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> list.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t is_col_vector(x): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>is_col_vector</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n\t radius:           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> radius
            <font face='Lucida Console'>)</font>;

        <font color='#0000FF'>return</font> <font color='#BB00BB'>find_min_trust_region</font><font face='Lucida Console'>(</font>stop_strategy,
                                     least_squares_lm_model<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>f, der, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>list<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
                                     x, 
                                     radius<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_OPTIMIZATION_LEAST_SQuARES_H___
</font>


</pre></body></html>